package com.terren.spider.core.html.pipeline;

import java.util.ArrayList;
import java.util.List;

import org.apache.log4j.Logger;

import com.terren.spider.core.html.configuration.Configuration;
import com.terren.spider.core.html.util.HtmlParseUtil;
import com.terren.spider.core.html.util.HtmlSaveUtil;
import com.terren.spider.db.service.SpiderLogService;
import com.terren.spider.util.common.LogUtil;

import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.pipeline.CollectorPipeline;

public abstract class BasicHtmlCollectorPipeline<T> implements CollectorPipeline<ResultItems> { 
	protected Logger log = LogUtil.getLogger(getClass());
	protected List<ResultItems> collector = new ArrayList<ResultItems>();
	protected Class<T> clazz;
	protected HtmlParseUtil<T> parseUtil;//分析util
	protected HtmlSaveUtil saveUtil;//存储util
	protected Integer saveCount;
	protected Configuration configuration;
	protected List<T> collectItems;
	
	protected SpiderLogService spiderLogService;// 爬虫日志service
	
	public BasicHtmlCollectorPipeline(Configuration configuration, Class<T> clazz) throws Exception{
		super();
		this.clazz = clazz;
		this.configuration = configuration;
		parseUtil = new HtmlParseUtil<T>(configuration);
		saveUtil = new HtmlSaveUtil();
		saveCount = 0;
		collectItems = new ArrayList<T>();
	}

	
	
	/**
	 * 批量保存到数据库
	 * @param collectItems
	 * @param totalPage 爬取的所有条目数
	 */
	protected void saveToDB(List<T>collectItems,Integer totalPage) throws Exception{
		//log.info(collectItems.size() + "-----" + totalPage + "----"+parseUtil.getParseCount());
		if (collectItems.size() > 0) {
			if (configuration.getSaveBatch()) {//批处理
				/*if (collectItems.size() == totalPage - saveCount) {
					saveUtil.saveBatch(collectItems,entry);
					saveCount = saveCount + collectItems.size();
					collectItems.clear();
					log.info("total save ["+saveCount+"]");
					return;
				}*/
				if (collectItems.size() % configuration.getSaveBatchCount() ==0||collectItems.size() > configuration.getSaveBatchCount()) {
					saveUtil.saveBatch(collectItems,configuration.getEntry());
					saveCount = saveCount + collectItems.size();
					collectItems.clear();
					return;
				}
			}
			else {
				saveUtil.saveBatch(collectItems,configuration.getEntry());
				collectItems.clear();
			}
				
		}
	}
	
    @Override
    public List<ResultItems> getCollected() {
        return collector;
    }

	public Class<T> getClazz() {
		return clazz;
	}

	public void setClazz(Class<T> clazz) {
		this.clazz = clazz;
	}

	public Configuration getConfiguration() {
		return configuration;
	}

	public void setConfiguration(Configuration configuration) {
		this.configuration = configuration;
	}

	public HtmlParseUtil<T> getParseUtil() {
		return parseUtil;
	}

	public void setParseUtil(HtmlParseUtil<T> parseUtil) {
		this.parseUtil = parseUtil;
	}

	public HtmlSaveUtil getSaveUtil() {
		return saveUtil;
	}

	public void setSaveUtil(HtmlSaveUtil saveUtil) {
		this.saveUtil = saveUtil;
	}

	public Integer getSaveCount() {
		return saveCount;
	}

	public void setSaveCount(Integer saveCount) {
		this.saveCount = saveCount;
	}

	public List<T> getCollectItems() {
		return collectItems;
	}

	public void setCollectItems(List<T> collectItems) {
		this.collectItems = collectItems;
	}
	
}
